<?php

define("IN", "H:/parse/src" );
define("OUT", "H:/parse/out");
define("INITIALIZATION_ERROR", -1);
define("FILE_ERROR", -3);
ini_set('max_execution_time','0');
ini_set('error_reporting', E_ALL);
ini_set('log_errors', '1');
ini_set('display_errors', '1');
$url_list = file(IN.'/list.txt');
$ch = curl_init($url);
$fileCounter = 0;
$logFilePath = OUT.'/log.txt';
echo $logFilePath;
$log = null;
$log = @fopen ($logFilePath, 'w');
if(!$log)print("Warning: log file is not created<br/>");
foreach ($url_list as $i){
    $url=trim($i);
    $fileCounter++;
    $f = null;
    $f = @fopen(OUT.'/'.$fileCounter.'.txt','w');
    $ch = curl_init($url);
    write_log($log, "Scanning\t".$i);
    $ex = parse_this($ch, $url, $f);
    if(!$ex){
        print "Error: $ex";
        break;
    }
    write_log($log, 'Process this url:'.'\t'.$url );
    if(!is_null($ch))curl_close($ch);
    if(!is_null($f) )fclose($f);
}
    
    function parse_this($ch, $url, $file = null){
    if($ch == null)return INITIALIZATION_ERROR;
	curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/2.0 (compatible; MSIE 2.1; Mac_PowerPC) ');
	curl_setopt($ch, CURLOPT_HEADER, 0);
    curl_setopt ($ch, CURLOPT_RETURNTRANSFER, 1);
    //curl_setopt($ch, CURLOPT_PROXY, '127.0.0.1:9050'); 
	if (!is_null($file))
        curl_setopt($ch, CURLOPT_FILE, $file);
    else return FILE_ERROR;
		curl_exec($ch);
	}
    
function write_log($log, $str){
     $s = date('d.m.y H:i:s').'\t'.$str.'\n';
      if(!is_null($log)){
         fputs($log, $s);
     }
}

 ?>